﻿Imports System.Web

Module Module1

    Sub Main()
        Try

            Dim targetPath As String = "C:\dev\MikeExport\DataFrom_PdfExpert\" ' With Trailing slash \
            Dim source As String = "http://192.168.1.2" ' Without Trailing slash

            Dim wc As New Net.WebClient


            RecurseFolder(wc, source, targetPath, source, targetPath)

            Console.ReadLine()


        Catch ex As Exception
            Console.WriteLine(ex.Message)
        End Try
        Console.ReadLine()

    End Sub
    Private Sub RecurseFolder(wc As Net.WebClient, source As String, targetPath As String, originalSource As String, originalTarget As String)
        If Not IO.Directory.Exists(targetPath) Then IO.Directory.CreateDirectory(targetPath)
        Dim cacheFile As String = targetPath & "root.xml"

        Dim s As String = Nothing
        If IO.File.Exists(cacheFile) Then
            s = IO.File.ReadAllText(cacheFile)
        Else
            s = DownloadString(wc, source)
            IO.File.WriteAllText(cacheFile, s)
        End If
        's = s.ToLower

        Dim urls As List(Of String) = GetUrls(s)
        Dim dirs As New List(Of String)(urls.Count)
        Dim files As New List(Of String)(urls.Count)
        Dim isDir As Boolean = True
        For Each i As String In urls
            isDir = Not i.Contains(".")
            If isDir Then dirs.Add(i) Else files.Add(i)
        Next

        For Each i As String In files
            Dim name As String = HttpUtility.UrlDecode(i.Substring(1))
            Dim filePath As String = originalTarget & name
            Dim fileUrl As String = originalSource & i
            DownloadFile(wc, fileUrl, filePath)
        Next
        For Each i As String In dirs
            Dim name As String = HttpUtility.UrlDecode(i.Substring(1))
            RecurseFolder(wc, source & i, targetPath & name & "\", originalSource, originalTarget)
        Next

    End Sub

    Private Function GetUrls(s As String, Optional startAt As Integer = 0, Optional list As List(Of String) = Nothing) As List(Of String)
        If IsNothing(list) Then list = New List(Of String)

        Dim startTag As Integer = s.IndexOf("<a ", startAt)
        If -1 = startTag Then Return list

        Dim startHref As Integer = s.IndexOf("href=""", startTag) + 6
        Dim endHref As Integer = s.IndexOf("""", startHref)
        Dim href As String = s.Substring(startHref, endHref - startHref)
        list.Add(href)

        Return GetUrls(s, endHref, list)
    End Function

    Private Function DownloadString(ByRef wc As Net.WebClient, url As String) As String
        Try
            Return wc.DownloadString(url)
        Catch ex As Exception
            Console.WriteLine("Failed to download url: " & url)
            Threading.Thread.Sleep(3000)
            Try
                Return wc.DownloadString(url)
            Catch ex2 As Exception
                Console.WriteLine("Failed to download url, spawning new webclient")
                wc.Dispose()

                wc = New Net.WebClient
                wc.Credentials = New Net.NetworkCredential("Polaris", "")
                Return DownloadString(wc, url)
            End Try
        End Try
    End Function
    Private Sub DownloadFile(ByRef wc As Net.WebClient, url As String, path As String)
        Try
            wc.DownloadFile(url.Replace("?", HttpUtility.UrlEncode("?")), path.Replace("?", "")) 'Filesystem doesnt like ?, and neither does the webbrowser (its own links are broken)
        Catch ex As Exception
            Console.WriteLine("Failed to download file: " & url)
            Threading.Thread.Sleep(3000)
            Try
                wc.DownloadFile(url, path)
            Catch ex2 As Exception
                Console.WriteLine("Failed to download file, spawning new webclient")
                wc.Dispose()

                wc = New Net.WebClient
                wc.Credentials = New Net.NetworkCredential("Polaris", "")
                DownloadFile(wc, url, path)
            End Try
        End Try
    End Sub


End Module
